import requests
from lxml import etree
import os
from concurrent.futures import ThreadPoolExecutor
import threading


def film_html(html_url):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'}
    html_response = requests.get(html_url, headers=headers)
    html_content = html_response.content.decode('utf8')
    html = etree.HTML(html_content)
    review_urls = html.xpath('//div[@class="main-bd"]/h2/a/@href')
    return review_urls


def get_review(review_url, file_path):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko'}
    try:
        review_html_response = requests.get(review_url, headers=headers)
        review_html_content = review_html_response.content.decode('utf8')
        review_html = etree.HTML(review_html_content)
        # 电影名
        movie_name = review_html.xpath('//div[@class="subject-title"]/a/text()')[0][2:]
        # 评论人和评分
        reviewer = review_html.xpath('//header/a/span/text()')[0]
        review_score = review_html.xpath('//header/span[1]/@title')[0]
        # 评论内容
        review_content_list = review_html.xpath('//div[@id="link-report"]/div/p//text()')
        with open(file_path, 'a', encoding='utf8') as f:
            f.write('url:%s\n' % review_url)
            f.write('电影名：%s\n' % movie_name)
            f.write('评论人：%s\n' % reviewer)
            f.write('评分：%s\n' % review_score)
            f.write('品论：\n')
            for review_content in review_content_list:
                f.write('%s\n' % review_content)
            f.write('*' * 50)
            f.write('\n')
        print('爬取成功：%s' % review_url)
    except Exception:
        print('失败：%s' % review_url)
        with open(file_path, 'a', encoding='utf8') as f:
            f.write('url:%s\n' % review_url)
            f.write('爬取失败\n')


if __name__ == '__main__':
    # 页面的rul
    html_urls = []
    for i in range(0, 5):
        i = i * 20
        temp_html_url = 'https://movie.douban.com/review/best/?start={}'.format(i)
        html_urls.append(temp_html_url)
    # 文件名
    file_name = os.getcwd() + os.sep + 'p06_douban_film_review_threading.txt'
    if os.path.exists(file_name):
        os.remove(file_name)

    # 获取页面中网评url
    with ThreadPoolExecutor() as html_pool:
        all_review_urls = html_pool.map(film_html, html_urls)

    # en。。。ip被屏蔽了，无法获取数据
    with ThreadPoolExecutor() as review_pool:
        for review_urls in all_review_urls:
            for url in review_urls:
                review_pool.submit(get_review, url, file_name)
